package com.ajaxw.crawler;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mozilla.intl.chardet.nsDetector;
import org.mozilla.intl.chardet.nsICharsetDetectionObserver;
import org.mozilla.intl.chardet.nsPSMDetector;

public class Crawler implements nsICharsetDetectionObserver {
	private String charset;

	public Crawler() {

		HttpURLConnection.setFollowRedirects(false);
	}

	public Map<String, String> getPage(String url) throws IOException {
		URLConnection conn = new URL(url).openConnection();
		HttpURLConnection hconn = (HttpURLConnection) conn;
		hconn.setRequestProperty("Accept-Encoding", "gzip");
		int timeout = 15 * 1000;
		hconn.setConnectTimeout(timeout);
		hconn.setReadTimeout(timeout);
		hconn.connect();

		int code = hconn.getResponseCode();
		if (code != 200)
			return null;

		// InputStream is = hconn.getInputStream();
		// BufferedInputStream bis = new BufferedInputStream(is);
		// Reader r = new InputStreamReader(bis, "ISO-8859-1");
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		int b = -1;
		// while (-1 != (b = r.read())) {
		// baos.write((byte) b);
		// }

		String a_encoding = hconn.getContentEncoding();

		// System.out.println(a_encoding);
		// System.out.println(t_encoding);
		// Map<String, List<String>> hdrs = hconn.getHeaderFields();
		String rsp = "";
		int i = 0;
		while (hconn.getHeaderField(i) != null) {
			rsp += hconn.getHeaderFieldKey(i) + ": " + hconn.getHeaderField(i)
					+ "\n";
			i++;
		}

		if (a_encoding != null && a_encoding.equalsIgnoreCase("gzip")) {
			GZIPInputStream gzipis = new GZIPInputStream(hconn.getInputStream());
			baos.reset();
			while ((b = gzipis.read()) != -1)
				baos.write((byte) b);
		} else {
			InputStream is = hconn.getInputStream();
			while (-1 != (b = is.read()))
				baos.write((byte) b);
		}

		// System.out.println(baos.toString());
		// System.out.println("-------------------------------------------------");
		// System.out.println(new String(baos.toByteArray(), "iso-8859-1"));

		// String charset = "UTF-8";
		String charset = null;
		Document doc = Jsoup.parse(baos.toString());
		// Elements metas = doc.select("meta[http-equiv=Content-Type]");
		Elements metas = doc.select("meta");

		Pattern p = Pattern.compile("charset=[\"]?([0-9a-zA-Z_\\-]+)[\"]?");
		Matcher m;

		for (Element meta : metas) {
			m = p.matcher(meta.toString());
			if (m.find())
				charset = m.group(1);
		}

		System.out.println(charset);

		this.charset = null;
		if (!"gb2312".equalsIgnoreCase(charset)) {
			detectCharset(baos);
		}

		if (this.charset != null /* && charset == null */)
			charset = this.charset;

		if (charset == null)
			charset = "gbk";

		// System.out.println(this.charset + ", " + charset);

		String str = new String(baos.toByteArray(), charset);
		str = Jsoup.parse(str).body().text();

		Map<String, String> rst = new HashMap<String, String>();
		rst.put("enc", charset);
		rst.put("txt", str);
		rst.put("rsp", rsp);
		return rst;
	}

	private void detectCharset(ByteArrayOutputStream baos) {
		nsDetector det = new nsDetector(nsPSMDetector.CHINESE);
		det.Init(this);
		det.DoIt(baos.toByteArray(), baos.size(), false);
		det.DataEnd();
	}

	@Override
	public void Notify(String charset) {
		this.charset = charset;
	}

	// private HttpURLConnection redirect(HttpURLConnection hconn)
	// throws IOException {
	// String loc = hconn.getHeaderField("location");
	// if (!loc.startsWith("http")) {
	// URL url = hconn.getURL();
	// loc = url.getProtocol() + "://" + url.getHost() + loc;
	// }
	// URLConnection conn = new URL(loc).openConnection();
	// hconn = (HttpURLConnection) conn;
	// hconn.connect();
	//
	// return hconn;
	// }
}
